﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using System.Text;
using System.Threading.Tasks;

namespace Crawer
{
  
    public class HtmlHelper
    {
        public static string  DownloadUrl(string url)
        {
            return DownLoadHtml(url, Encoding.UTF8);
        }

        public static string DownLoadHtml(string Url,Encoding encoding)
        {
            //定义一个空字符串
            string html = string.Empty;
            //使用HttpWebRequet.Creare
            try
            {
                //证书认证
                //ServicePointManager.ServerCertificateValidationCallback=new System.Net.Security.RemoteCertificateValidationCallback((object sender,X509Certificate certificate,X509Chain chain,SslPolicyErrors errors))=>{
                //    return true;
                //}
                HttpWebRequest request = HttpWebRequest.Create(Url) as HttpWebRequest;
                request.Timeout = 30 * 1000;//设置30S超时时间
                request.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36";
                request.ContentType = "text/html; charset=UTF-8";//gbk
                //Encoding enc = Encoding.GetEncoding("GB2312");
                request.CookieContainer = new CookieContainer();//请求Cookie容器

                using (HttpWebResponse response =request.GetResponse() as HttpWebResponse)
                {
                    if (response.StatusCode!=HttpStatusCode.OK)
                    {
                        Console.WriteLine(string.Format("抓取地址{0}失败，request.StatusCode为{1}", Url, response.StatusCode));
                    }
                    else
                    {
                        try
                        {
                           // string cookvalue = response.Cookies["session_id"].Value;
                            //成功请求获取内容
                            StreamReader st = new StreamReader(response.GetResponseStream(), encoding);
                            html = st.ReadToEnd();
                            st.Close();
                         }
                        catch (Exception ex)
                        {
                            Console.WriteLine(string.Format($"读取失败抓取{Url}失败"), ex);
                            throw;
                        }
                    }
                }
            }
            catch (Exception ex)
            {

                throw ex;
            }

            return html;
        }


    }
}
